# -*- python -*-
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import json
import hashlib
import os
import platform
import sys

Import('env')

# We only want to export validator functions when we build with scons, because
# they are used for tests.
# On the other hand, exporting unnecessary symbols causes chromium build
# problems, see https://code.google.com/p/nativeclient/issues/detail?id=3367 ,
# so we leave VALIDATOR_EXPORT undefined in gyp build.
env = env.Clone()
env.Append(CCFLAGS=['-DVALIDATOR_EXPORT=DLLEXPORT'])

# Dir to place c files generated by 'dfagen' target (it's under source control).
gen_dir = '$MAIN_DIR/src/trusted/validator_ragel/gen'
# Currently we only provide ragel pre-built for Linux.
ragel_binary = '$MAIN_DIR/../third_party/ragel/ragel.linux'

INSTRUCTION_DEFINITIONS = map(env.File, [
    'instruction_definitions/general_purpose_instructions.def',
    'instruction_definitions/system_instructions.def',
    'instruction_definitions/x87_instructions.def',
    'instruction_definitions/mmx_instructions.def',
    'instruction_definitions/xmm_instructions.def',
    'instruction_definitions/nops.def',
    'instruction_definitions/tls.def',
])

# These objects are included in both dfa_validate_x86_xx and rdfa_validator
# libraries, so we have to introduce intermediate scons nodes.
validator32 = env.ComponentObject('gen/validator_x86_32.c')
validator64 = env.ComponentObject('gen/validator_x86_64.c')

features = [
    env.ComponentObject('validator_features_all.c'),
    env.ComponentObject('validator_features_validator.c')
]

# Glue library called from service runtime. The source file depends on the
# target architecture.  In library_deps.py this library is marked as
# dependant of dfa_validate_x86_xx.
if env.Bit('build_x86'):
  caller_lib = 'dfa_validate_caller_x86_%s' % env.get('TARGET_SUBARCH')
  env.ComponentLibrary(
      caller_lib,
      ['dfa_validate_%s.c' % env.get('TARGET_SUBARCH'),
       {'32': validator32, '64': validator64}[env.get('TARGET_SUBARCH')],
       'dfa_validate_common.c',
       features])

# Low-level platform-independent interface supporting both 32 and 64 bit,
# used in ncval and in validator_benchmark.
env.ComponentLibrary('rdfa_validator',
                     [validator32, validator64] + features)

validator_benchmark = env.ComponentProgram(
    'rdfa_validator_benchmark',
    ['validator_benchmark.cc'],
    EXTRA_LIBS=['rdfa_validator', 'platform', 'elf_load']
)

run_benchmark = env.AutoDepsCommand(
    'run_validator_ragel_benchmark.out',
    [validator_benchmark, env.GetIrtNexe(), '10000']
)

env.AlwaysBuild(env.Alias('dfavalidatorbenchmark', run_benchmark))

# We don't run this test under qemu because it attempts to execute host python
# binary.
gen_dfa_test = env.CommandTest(
    'gen_dfa_test.out',
    ['${PYTHON}', env.File('gen_dfa_test.py')] + INSTRUCTION_DEFINITIONS,
    direct_emulation=False)

env.AddNodeToTestSuite(
    gen_dfa_test,
    ['small_tests', 'validator_tests'],
    'run_gen_dfa_test')

trie_test = env.CommandTest(
    'trie_test.out',
    ['${PYTHON}', env.File('trie_test.py'), '--verbose'],
    direct_emulation=False)

env.AddNodeToTestSuite(
    trie_test,
    ['small_tests', 'validator_tests'],
    'run_trie_test')

proof_tools_test = env.CommandTest(
    'proof_tools_test.out',
    ['${PYTHON}', env.File('proof_tools_test.py'), '--verbose'],
    direct_emulation=False)

env.AddNodeToTestSuite(
    proof_tools_test,
    ['small_tests', 'validator_tests'],
    'run_proof_tools_test')

dll_env = env.Clone(COMPONENT_STATIC=False)

if env.Bit('windows'):
  # On windows we don't need to recompile specifically for dynamic linking.
  validator32_dll = validator32
  validator64_dll = validator64
  features_dll = features
else:
  dll_env.Append(CCFLAGS=['-fPIC'])
  validator32_dll = dll_env.ComponentObject('gen/validator_x86_32.c')
  validator64_dll = dll_env.ComponentObject('gen/validator_x86_64.c')

  features_dll = [
      dll_env.ComponentObject('validator_features_all.c'),
      dll_env.ComponentObject('validator_features_validator.c')
  ]

dll_utils = dll_env.ComponentObject('dll_utils.c')

validator_dll = dll_env.ComponentLibrary(
    'rdfa_validator_dll',
    [validator32_dll, validator64_dll, dll_utils] + features_dll)

# Here for simplicity we make assumption that python used to run scons
# is the same as the one invoked by scons to run tests.
python_bitness = {2**31 - 1: 32, 2**63 - 1: 64}[sys.maxsize]
bitness = int(env.get('TARGET_SUBARCH'))
assert bitness in [32, 64]
# Note that it is possible that we build on one machine and run tests on another
# one, so we build DLL even if python bitness does not match target bitness.
# Asan does not instrument DLLs (only executables), so we will be unable to
# run tests under asan.
python_can_load_dll = python_bitness == bitness and not env.Bit('asan')

# On ARM we still have some test bots that are running an armel system
# image (with armhf runtime supported added) which means that the python
# executable is armel and cannot load binaries built by the arm cross
# compiler that we use.
# TODO(sbc): remove this once the testbots get upgraded to armhf:
# https://code.google.com/p/nativeclient/issues/detail?id=3716
if platform.machine() == 'armv7l':
  with open(sys.executable) as infile:
    if '/lib/ld-linux-armhf' not in infile.read(1024):
      python_can_load_dll = False

if python_can_load_dll:
  validator_py_test = env.AutoDepsCommand(
        'validator_py_out',
        ['${PYTHON}',
         env.File('validator.py'),
         validator_dll])

  env.AddNodeToTestSuite(
      validator_py_test,
      ['small_tests', 'validator_tests'],
      'run_validator_py_test')

# Source generation:
#
#   dfagen : Regenerate any autogenerated source files.

ragel_targets = set([
    'dfagen',
    'dfacheckdecoder',
    'dfacheckvalidator',
    'dfagentries',
    'dfachecktries'])
ragel_involved = ragel_targets.intersection(COMMAND_LINE_TARGETS)

gas = env.MakeUntrustedNativeEnv()['AS']
objdump = env.MakeUntrustedNativeEnv()['OBJDUMP']

if ragel_involved:
  if not env.Bit('host_linux'):
    raise Exception('Right now DFA generation is only supported on Linux')

  # Source generation step 1: Build generator of ragel files.
  #
  # We have generator which reads .def files and produced automaton definition.
  #
  # Ragel is included in most Linux distributions, but it's not standard tool
  # on MacOS/Windows thus we only support gneration of automata under Linux.
  # This also means that we don't need to make sure gen_dfa.cc is portable to
  # non-POSIX platforms (in particular it's not Windows compatible).

  env_gen_dfa = env.Clone()
  env_gen_dfa.Append(CCFLAGS=['-DNACL_TRUSTED_BUT_NOT_TCB'])

  # Generate byte_machines.rl from byte_machines.py
  (byte_machines,) = env.AutoDepsCommand(
      'byte_machines.rl',
      ['${PYTHON}',
       env.File('byte_machines.py'),
       '>${TARGET}'])

  # Source generation step 2: Generate decoder automata.
  #
  # Now we are back to conditionally defining the large automata generated
  # by gen_dfa.

  def MakeAutomaton(bits, automaton):
    (rl_instruction_file,) = env.AutoDepsCommand(
        '%s_x86_%s_instruction.rl' % (automaton, bits),
        ['${PYTHON}',
         env.File('gen_dfa.py'),
         '--bitness', bits,
         '--mode', automaton,
         '>${TARGET}',
         ] + INSTRUCTION_DEFINITIONS)

    include_dir = rl_instruction_file.dir.get_abspath()
    assert include_dir == byte_machines.dir.get_abspath()

    # Don't use env.File because filename is written "as is" in XML file.
    # We want relative path there, not an absolute one!
    rl_file = 'src/trusted/validator_ragel/%s_x86_%s.rl' % (automaton, bits)

    (xml_file,) = env.AutoDepsCommand(
        '%s/%s_x86_%s.xml' % (gen_dir, automaton, bits),
        [ragel_binary,
         '-x',
         '-LL',
         '-I..',
         '-I%s' % include_dir,
         rl_file,
         '-o', '${TARGET}'],
        extra_deps=[byte_machines, rl_instruction_file])

    c_file = env.AutoDepsCommand(
        '%s/%s_x86_%s.c' % (gen_dir, automaton, bits),
        ['${PYTHON}',
         env.File('codegen.py'),
         rl_file,
         xml_file,
         '${TARGET}'])
    # We need to at least update timestamp if dfagen is started.
    # It is needed for presubmit script.
    env.AlwaysBuild(c_file)

    return c_file, xml_file

  (decoder32, decoder32_xml) = MakeAutomaton(
      '32', 'decoder')
  (validator32, validator32_xml) = MakeAutomaton(
      '32', 'validator')
  (decoder64, decoder64_xml) = MakeAutomaton(
      '64', 'decoder')
  (validator64, validator64_xml) = MakeAutomaton(
      '64', 'validator')

  automata = [decoder32, validator32, decoder64, validator64]

  # Prepair 'dfacheckdecoder' test.
  #
  # In this test, all acceptable instructions are enumerated
  # by DFA traversal, and for each one objdump output and
  # DFA-based decoder output are compared.
  # It takes few hours to run the test, so it's not included
  # into any suits and is supposed to be run manually when
  # changes are made to DFA definitions.
  # Also, since DFA generation is currently linux-only,
  # this test is somewhat platform-dependent as well.

  decoder32_obj = dll_env.ComponentObject(decoder32)
  decoder64_obj = dll_env.ComponentObject(decoder64)
  decoder_test_obj = dll_env.ComponentObject('unreviewed/decoder_test.cc')

  # Command-line decoder.
  decoder_test = env.ComponentProgram(
      'decoder_test',
      [decoder_test_obj, decoder32_obj, decoder64_obj])

  decoder_dll = dll_env.ComponentLibrary(
      'rdfa_decoder_dll',
      [decoder_test_obj, decoder32_obj, decoder64_obj])

  check_decoders = []

  for bits, xml in [('32', decoder32_xml), ('64', decoder64_xml)]:
    check_decoder = env.AutoDepsCommand(
        'check_decoder_test_results_%s' % bits,
        ['${PYTHON}',
         env.File('check_decoder.py'),
         xml,
         '--gas', gas,
         '--objdump', objdump,
         '--decoder', decoder_test,
         '--bits', bits])

    check_decoders.append(check_decoder)

  # Never run decoder tests in parallel because they can take all CPU.
  SideEffect('check_decoder', check_decoders)

  env.AlwaysBuild(env.Alias('dfagen', automata))
  env.AlwaysBuild(env.Alias('dfacheckdecoder', check_decoders))

  gen_tries = []
  check_tries = []

  for bits, xml in [('32', validator32_xml), ('64', validator64_xml)]:
    latest_trie_file = env.AutoDepsCommand(
        '%s/gen_trie_%s.out' % (gen_dir, bits),
        ['${PYTHON}',
         env.File('gen_trie_from_dfa.py'),
         xml,
         '--bitness', bits,
         '--validator_dll', validator_dll,
         '--decoder_dll', decoder_dll,
         '--trie_path', '${TARGET}'])
    gen_tries.append(latest_trie_file)

    check_latest_trie_file = env.AutoDepsCommand(
        'check_latest_trie_%s.out' % bits,
        ['${PYTHON}',
         env.File('check_trie.py'),
         '--bitness', bits,
         latest_trie_file])
    check_tries.append(check_latest_trie_file)

  SideEffect('gen_tries', gen_tries)
  env.Alias('dfagentries', gen_tries)

  snapshots_dir = '${MAIN_DIR}/../validator_snapshots'
  trie_proofs = [
    # (proof_file_name, (bitnesses), old_trie_path, new_trie_path)
    ('vzero_proof.py', ('32', '64'), '20141001', '20141020.vzero'),
    ('movbe_proof.py', ('32', '64'), '20141020.vzero', '20141021.movbe'),
    ('bsr_bsf_proof.py', ('32', '64'), '20141021.movbe', '20141027.bsr_bsf'),
    ('shld_shrd_proof.py', ('32', '64'), '20141027.bsr_bsf',
     '20141028.shld_shrd'),
    ('xadd16_proof.py', ('32', '64'), '20141028.shld_shrd', '20141029.xadd16'),
    ('cmpxchg16_proof.py', ('32', '64'), '20141029.xadd16',
     '20141030.cmpxchg16'),
    ('avx1_xmm_ymm_memory_3op_pd_proof.py', ('32', '64'), '20141030.cmpxchg16',
     '20141031.avx1_3op_fp_pd'),
    ('avx1_xmm_ymm_memory_3op_ps_proof.py', ('32', '64'),
     '20141031.avx1_3op_fp_pd', '20141104.avx1_3op_fp_ps'),
    ('avx1_xmm_memory_3op_pi_proof.py', ('32', '64'),
     '20141104.avx1_3op_fp_ps', '20141105.avx1_3op_pi'),
    ('avx1_xmm_memory_3op_pb_proof.py', ('32', '64'),
     '20141105.avx1_3op_pi', '20141106.avx1_3op_pb'),
    ('avx1_xmm_memory_3op_fp_scalars_proof.py', ('32', '64'),
      '20141106.avx1_3op_pb', '20141110.avx1_3op_fp_scalars'),
    ('vbroadcast_proof.py', ('32', '64'),
     '20141110.avx1_3op_fp_scalars', '20141115.avx1_vbroadcast'),
    ('avx1_3op_simd_or_memory_others_proof.py', ('32', '64'),
     '20141115.avx1_vbroadcast', '20141118.avx1_remaining_3operand'),
    ('avx1_xmm_ymm_memory_moves.py', ('32', '64'),
     '20141118.avx1_remaining_3operand', '20141119.avx1_moves'),
    ('remaining_nonspecial_avx1_proof.py', ('32', '64'),
     '20141119.avx1_moves', '20141204.all_nonspecial_avx1'),
    ('final_avx1_instrs_proof.py', ('32', '64'),
     '20141204.all_nonspecial_avx1', '20141210.remaining_avx1'),
    ('fma_proof.py', ('32', '64'), '20141210.remaining_avx1', '20141215.fma'),
    ('vpminsd_avx2_proof.py', ('32', '64'),
     '20141215.fma', '20150106.vpminsd.avx2'),
    ('avx2_0f_6x_0f_7x_proof.py', ('32', '64'),
     '20150106.vpminsd.avx2', '20150107.avx2.groups_0f_6x_0f_7x'),
    ('remaining_avx2_promotions_proof.py', ('32', '64'),
     '20150107.avx2.groups_0f_6x_0f_7x', '20150113.remaining_avx2_promotions'),
    ('avx2_additions_proof.py', ('32', '64'),
     '20150113.remaining_avx2_promotions', '20150128.avx2_additions'),
    ('crc32_16_to_32_proof.py', ('32', '64'),
     '20150210.vmovq_bugfix', '20150223.crc32w'),
    ('jecxz_proof.py', ('32', '64'),
     '20150223.crc32w', '20150226.jecxz'),
    ('vmovd_proof.py', ('32', '64'),
     '20150226.jecxz', '20150430.vmovd_bugfix'),
    ('nacl_unsupported_proof.py', ('32', '64'),
     '20150430.vmovd_bugfix', '20150723.nacl_unsupported'),
  ]
  # Run the 5 latest proofs by default.
  for proof_file, bitnesses, old, new in trie_proofs[-5:]:
    for bitness in bitnesses:
      trie_subdirs = {'32': 'ragel_trie_x86_32', '64': 'ragel_trie_x86_64'}
      check_proof = env.AutoDepsCommand(
        '%s_%s_results' % (proof_file, bitness),
        ['${PYTHON}',
         env.File(proof_file),
         '--validator_dll', validator_dll,
         '--decoder_dll', decoder_dll,
         '--bitness', bitness,
         '--gas', gas,
         '--objdump', objdump,
         '--old', snapshots_dir + '/' + trie_subdirs[bitness] + '/' + old,
         '--new', snapshots_dir + '/' + trie_subdirs[bitness] + '/' + new])
      check_tries.append(check_proof)

  # Don't run gen_trie tests in parallel because they can take all CPU.
  SideEffect('check_tries', check_tries)
  env.Alias('dfachecktries', check_tries)

  if python_can_load_dll:

    dfacheckvalidator_tests = []

    for bitness, xml in [('32', validator32_xml), ('64', validator64_xml)]:
      superinstruction = env.AutoDepsCommand(
          'superinstructions_x86_%s.txt' % bitness,
          ['${PYTHON}',
           env.File('verify_validators_dfa.py'),
           xml,
           '-o', '${TARGET}'])

      superinstruction_verified = env.AutoDepsCommand(
        'superinstructions_verified_x86_%s.out' % bitness,
        ['${PYTHON}',
         env.File('verify_superinstructions.py'),
         '--bitness', bitness,
         '--gas', gas,
         '--objdump', objdump,
         '--validator_dll', validator_dll,
         superinstruction])

      env.AlwaysBuild(superinstruction_verified)
      dfacheckvalidator_tests.append(superinstruction_verified)

      regular_instructions_test = env.AutoDepsCommand(
        'regular_instructions_test_x86_%s.out' % bitness,
        ['${PYTHON}',
         env.File('verify_regular_instructions.py'),
         xml,
         '--bitness', bitness,
         '--validator_dll', validator_dll,
         '--decoder_dll', decoder_dll])

      # Never run exhaustive tests in parallel because they can take all CPU.
      env.AlwaysBuild(regular_instructions_test)
      SideEffect('check_validator', regular_instructions_test)
      dfacheckvalidator_tests.append(regular_instructions_test)

    # There are three categories of protected files: validator files, generated
    # files (those of validator files that lie withing 'gen' directory) and
    # generating files (programs and input data used to produce generated
    # files).
    #
    # When validator files are changed, exhaustive validator tests have to be
    # rerun.  When generating files are changed, dfagen must be run to ensure
    # that generated files are up to date. Presubmit script uses
    # protected_files.json to perform these checks.

    # Files which directly affect the validator behavior.
    protected_files = [
      'gen/validator_x86_32.c',
      'gen/validator_x86_32.xml',
      'gen/validator_x86_64.c',
      'gen/validator_x86_64.xml',
      'validator_internal.h',
      'decoder.h',
      'decoding.h',
      'validator.h']

    # Files which are used to generate gen/{decoder,validator}_x86_{32,64}.c
    # files.
    generating_files = [
      INSTRUCTION_DEFINITIONS,
      'validator_x86_32.rl',
      'validator_x86_64.rl',
      'parse_instruction.rl',
      'byte_machines.py',
      'gen_dfa.py',
      'codegen.py']

    def CanonicalizeFilename(filename):
      return 'native_client/' + env.File(filename).get_path().replace('\\', '/')

    def CalculateHashSums(target, source, env):
      target_filename = target[0].get_abspath()
      protected_files = {
        'validator':{},
        'generated':[],
        'generating':[]
      }
      for filename in source:
        canonical_filename = CanonicalizeFilename(filename)
        with open(filename.get_abspath(), 'r') as file_stream:
          sha512 = hashlib.sha512(file_stream.read()).hexdigest()
          protected_files['validator'][canonical_filename] = sha512
        if canonical_filename.rsplit('/', 2)[1] == 'gen':
          protected_files['generated'].append(canonical_filename)
      for filename in Flatten(generating_files):
        canonical_filename = CanonicalizeFilename(filename)
        protected_files['generating'].append(canonical_filename)
      with open(target_filename, 'w') as target_file:
        json.dump(protected_files, target_file, sort_keys=True, indent=2)

    calculate_hash_sums = env.Command(
        target='%s/protected_files.json' % gen_dir,
        source=protected_files,
        action=CalculateHashSums)
    env.Depends(calculate_hash_sums, dfacheckvalidator_tests)

    env.Alias('dfacheckvalidator', calculate_hash_sums)

# Targeted tests: RDFA validator test, dis section checker and spec_val test.

for bits in ['32', '64']:
  tests_mask = (
      '${MAIN_DIR}/src/trusted/validator_ragel/testdata/%s/*.test' % bits)

  if env.Bit('regenerate_golden'):
    update_option = ['--update']
  else:
    update_option = []

  if env.Bit('build_x86'):
    # In principle we can run these tests on ARM too, but first, there is no
    # reason to, and second, there are problems with those bots that build
    # on one machine and run tests on another (build machine does not recognize
    # than new ncval is needed, so it's not copied to the test machine).
    rdfaval = '$STAGING_DIR/ncval_new$PROGSUFFIX'
    rdfa_targeted_test = env.AutoDepsCommand(
        'run_rdfa_targeted_tests_%s.out' % bits,
        ['${PYTHON}',
         env.File('run_rdfa_validator_tests.py'),
         '--rdfaval', env.File(rdfaval),
         '--bits', bits,
         tests_mask] + update_option)

    env.AddNodeToTestSuite(
        rdfa_targeted_test,
        ['small_tests', 'validator_tests'],
        node_name='run_rdfa_targeted_tests_%s' % bits)

    if env.Bit('regenerate_golden'):
      # Don't want these tests run in parallel because they write
      # to .test files.
      SideEffect(tests_mask, rdfa_targeted_test)

    dis_section_test = env.AutoDepsCommand(
        'run_dis_section_test_%s.out' % bits,
        ['${PYTHON}',
         env.File('check_dis_section.py'),
         '--objdump', objdump,
         '--bits', bits,
         tests_mask] + update_option)

    env.AddNodeToTestSuite(
        dis_section_test,
        ['small_tests', 'validator_tests'],
        node_name='run_dis_section_test_%s' % bits)

    if env.Bit('regenerate_golden'):
      # Don't want these tests run in parallel because they write
      # to .test files.
      SideEffect(tests_mask, dis_section_test)

  spec_test = env.AutoDepsCommand(
      'run_spec_val_test_%s.out' % bits,
      ['${PYTHON}',
       env.File('spec_val_test.py'),
       '--bits', bits,
       tests_mask] + update_option)

  env.AddNodeToTestSuite(
      spec_test,
      ['small_tests', 'validator_tests'],
      node_name='run_spec_val_test_%s' % bits)

  if env.Bit('regenerate_golden'):
    # Don't want these tests run in parallel because they write
    # to .test files.
    SideEffect(tests_mask, spec_test)

    # Spec test uses @dis section in test files, which in turn is updated
    # by dis_section_test.
    if env.Bit('build_x86'):
      env.Depends(spec_test, dis_section_test)
